#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import string
import MySQLdb
from math import log
from sets import Set

def import_baike_tree():
    data = open('/home/chaojiansong/videobook/notinsvn/baike_tree', 'r')
    title = ''
    for line in data:
        m = re.match('title: (.*)', line)
        if m:
            title = unicode(m.group(1), 'utf-8', 'ignore').encode('utf-8')
            if title not in tree:
                tree[title] = []
        # baike_tree中可能存在一个title出现几次，分类不同的情况，在此合并
        else:
            m = re.match('cat[0-2]: (.*)', line)
            if m:
                cat = unicode(m.group(1), 'utf-8', 'ignore').encode('utf-8')
                if cat != '':
                    tree[title].append(cat)
            else:
                print 'regex error'
    data.close()
    # 加入上传者名字
    uploaders = open('/home/chaojiansong/videobook/notinsvn/uploader_list', 'r')
    for line in uploaders:
        uploader = line.rstrip()
        if uploader not in tree:
            tree[uploader] = []
    uploaders.close()

def map2baike(tag):
    ret = []
    tag = string.lower(tag)
    if tag in tree:
    	ret.append(tag)
   # tag_len = len(tag)
   # over = False
   # for i in range(0, tag_len - 4):
   #     for j in range(0, i + 1):
   #         cur_tag = tag[j:j + tag_len - i]
   #         if cur_tag in tree:
   #             over = True
   #             ret.append(cur_tag)
   #     if over:
   #         break
    return ret

def get_idf(tag, tag_df):
    df = 1.0
    if tag in tag_df:
        df = tag_df[tag] + 1.0
    return log(video_cnt / df)

def get_weight(t1, t2, tag_df):
    if t1 == t2:
        return 1
    if len(tree[t1]) == 0 or len(tree[t2]) == 0:
        return 0
    up = 0
    down = 0
    for cat_tag1 in tree[t1]:
        down = down + get_idf(cat_tag1, tag_df)
        for cat_tag2 in tree[t2]:
            if cat_tag2 == cat_tag1:
                up = up + get_idf(cat_tag1, tag_df)
                break
    #return up / down
    #不含有某相同tag很普遍，不能认为差很多加在分母上
    return up * get_idf(t1, tag_df)

def tagrank(graph, n, score):
    for i in range(0, n):
        for j in range(0, n):
            print graph[i][j],
            score[j] = score[j] + graph[i][j]
        print '\n',
    print '***',
    for i in range(0, n):
    	print score[i],
    print '\n',

def select_tags(tags):
    # 如果tag总数<=3，不进行筛选，直接作为最终tag
    if len(tags) <= 3:
        return
    # 将tag加入tree
    for tag in tags:
        if tag not in tree:
            tree[tag] = []
    # 1 读取tag及其分类tag的df信息，得到tag对应的df向量
    tag_df = {}
    tag_dfs = open('/home/chaojiansong/videobook/notinsvn/baike_tag_df', 'r')
    line_no = 0
    for line in tag_dfs:
        if line_no % 2 == 0:
            cur_tag = line.rstrip()
        else:
            tag_df[cur_tag] = int(line.rstrip())
    # 2 构图，边权v12＝(v1,v2相同tag idf之和) / v1的tag idf之和
    #             v21= (v1,v2相同tag idf之和) / v2的tag idf之和
    tag_cnt = len(tags)
    graph = [ [0 for x in range(0, tag_cnt)] for y in range(0, tag_cnt) ]
    damping_factor = 1
    for i in range(0, tag_cnt):
        for j in range(0, tag_cnt):
            graph[i][j] = damping_factor * get_weight(tags[i], tags[j], tag_df) + (1 - damping_factor)
    # 3 计算图的pagerank(tagrank?)选出排名靠的tag
    score = [0 for x in range(0, tag_cnt)]
    res = {}
    tagrank(graph, tag_cnt, score)
    print len(tags), len(score), tag_cnt
    for i in range(0, tag_cnt):
    	res[tags[i]] = score[i]
    tags = []
    print 'tag sorted:'
    for tag, score in sorted(res.items(), key = lambda x: x[1], reverse = True):
    	tags.append(tag)
    	print tag,
    print '\n',

# 得到视频总数，用于计算idf
conn = MySQLdb.Connection("172.16.7.58", "root", "researchsucks", "X", 3306)
conn.set_character_set("utf8");
cur = conn.cursor()
cur.execute("select count(distinct video_id) from pop_history_baike_tags")
record = cur.fetchone()
video_cnt = (float)(record[0])
# 读取stop word列表
stopwords = Set()
stopwords_list = open('/home/chaojiansong/videobook/notinsvn/stopwords', 'r')
for line in stopwords_list:
	stopwords.add(line.rstrip())
stopwords_list.close()
# 读取baike类别树
tree = {}
import_baike_tree()
# 将baike_tag去重
for key in tree:
    tree[key] = {}.fromkeys(tree[key]).keys()

# 读当天视频候选列表
data = open('/home/chaojiansong/videobook/notinsvn/video_score_list.uniq', 'r')
cnt = 0
video_ids = []
for line in data:
    if cnt % 6 == 0:
        video_ids.append(line.rstrip())
    cnt = cnt + 1
data.close()

# 得到视频tag对应的baike tag，写入pop_history_baike_tags
conn = MySQLdb.Connection("172.16.7.58", "root", "researchsucks", "X", 3306)
conn.set_character_set("utf8");
cur = conn.cursor()
cnt = 0
for id in video_ids:
    cur.execute("select tag from video_tags where video_id='" + id + "'")
    print '#####START#####'
    print 'id: ' + id
    # 筛选video_tag，去除噪音tag
    video_tags = []
    for record in cur.fetchall():
        video_tags.append(unicode(record[0], 'utf-8', 'ignore').encode('utf-8'))
    all_baike_tags = []
    for tag in video_tags:
        baike_tags = map2baike(tag)
        for t in baike_tags:
            if t not in stopwords:
                all_baike_tags.append(t)
#############################################
    print 'video tag: ',
    for tag in video_tags:
        print tag,
    print '\n',
    print 'baike tag: ',
    for baike_tag in all_baike_tags:
        print baike_tag,
    print '\n',
############################################
    #select_tags(all_baike_tags)
#############################################
    #print 'baike tag after selection:'
    #for baike_tag in all_baike_tags:
    #    print '\tbaike_tag: ' + baike_tag
    #    for cat_tag in tree[baike_tag]:
    #        print '\t\tcat_tag: ' + cat_tag
#############################################
    for baike_tag in all_baike_tags:
        cur.execute("replace into pop_history_baike_tags(video_id, baike_tag) values('" + id + "', '" + re.sub("'", "\\'", baike_tag) + "')")
        for cat_tag in tree[baike_tag]:
            cur.execute("replace into pop_history_baike_tags(video_id, baike_tag) values('" + id + "', '" + re.sub("'", "\\'", cat_tag) + "')")
    print '#####END#####'

cur.close()
conn.close()
